﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Xml;
using System.IO;
using porter;
using MyTestProject;

namespace ParseReuters21578
{
    public static class Reuters21578
    {
        static void Main(string[] args)
        {
            bool stem = true;
            bool removeStopWords = true;
            bool removeDigits = true;

            //List<Article> dataset = ReadFiles(stem, removeStopWords, removeDigits);

            //Dictionary<string, int> allTopics = GetAllTopics(dataset);

            //List<Article>[] modApt = GetModAptList(dataset);
        }

        static List<ICheckable>[] GetModAptList(List<Article> dataset)
        {
            //[0] - train; [1] - test
            List<ICheckable>[] ans = new List<ICheckable>[2];
            List<ICheckable> train = new List<ICheckable>();
            List<ICheckable> test = new List<ICheckable>();
            foreach (Article article in dataset)
            {
                if (article.HasTopics && article.Lewissplit == LEWISSPLIT.TRAIN)
                {
                    train.Add(article);
                }
                if (article.HasTopics && article.Lewissplit == LEWISSPLIT.TEST)
                {
                    test.Add(article);
                }
            }
            ans[0] = train;
            ans[1] = test;
            return ans;
        }

        public static List<ICheckable> GetCompleteModApte(List<ICheckable> dataset)
        {
            List<ICheckable> ans = new List<ICheckable>();
            Article article = null;
            foreach (ICheckable item in dataset)
            {
                if (item.GetType() == typeof(Article))
                {
                    article = (Article)item;
                    if (article.HasTopics && article.Lewissplit == LEWISSPLIT.TRAIN ||
                        article.HasTopics && article.Lewissplit == LEWISSPLIT.TEST)
                    {
                        ans.Add(article);
                    }
                }
                
            }
            return ans;
        }

        static Dictionary<string, int> GetAllTopics(List<Article> dataset)
        {
            Dictionary<string, int> topics = new Dictionary<string, int>();
            int counter = 0;
            foreach (Article article in dataset)
            {
                if (article.Topics == null)
                {
                    if (topics.ContainsKey("null"))
                    {
                        topics["null"]++;
                    }
                    else
                    {
                        topics.Add("null", 1);
                    }
                    if (article.HasTopics)
                    {
                        counter++;
                    }
                }
                else
                {
                    foreach (string topic in article.Topics)
                    {
                        if (topics.ContainsKey(topic))
                        {
                            topics[topic]++;
                        }
                        else
                        {
                            topics.Add(topic, 1);
                        }
                    }
                }
            }
            return topics;
        }

        public static List<ICheckable> ReadFiles(bool stem, bool removeStopWords, bool removeDigits)
        {
            DirectoryInfo di = new DirectoryInfo("reuters21578-xml");
            FileInfo[] files = di.GetFiles("*.xml");

            string id = string.Empty, date = string.Empty, author = string.Empty,
                body = string.Empty, title = string.Empty;
            LEWISSPLIT lewissplit;
            List<string> topics = null, places = null;
            bool hasTopics;
            string[] stopWords = null;

            if (removeStopWords)
            {
                stopWords = LoadStopWords();
            }

            char[] wrongChars = { '\n', '\r', '\t', ',', '.', ':', '\\', '!', '@', '#', '$', '%','?','<','>','\"','\'',
                                    '^', '&', '*', '(', ')', '[', ']', '{', '}', '_', '-', '=', '+', '/', ';' };
            if (removeDigits)
            {
                List<char> wrongCharList = new List<char>(wrongChars);
                string wrongString = new string(wrongChars);
                for (int j = 0; j < 10; j++)
                {
                    wrongString += j.ToString();
                }
                wrongChars = wrongString.ToCharArray();
            }


            List<ICheckable> dataset = new List<ICheckable>();

            XmlDocument xmlDoc = new XmlDocument();
            foreach (FileInfo fi in files)
            {
                xmlDoc.Load(fi.FullName);
                XmlElement root = xmlDoc.DocumentElement;
                int count = root.ChildNodes.Count;

                XmlNode article;
                for (int i = 0; i < count; i++)
                {
                    #region variable reset
                    hasTopics = false;
                    date = string.Empty;
                    author = string.Empty;
                    body = string.Empty;
                    title = string.Empty;
                    topics = null;
                    places = null;
                    #endregion

                    article = root.ChildNodes[i];
                    id = article.Attributes["NEWID"].Value;

                    switch (article.Attributes["LEWISSPLIT"].Value)
                    {
                        case "TRAIN":
                            lewissplit = LEWISSPLIT.TRAIN;
                            break;
                        case "TEST":
                            lewissplit = LEWISSPLIT.TEST;
                            break;
                        default:
                            lewissplit = LEWISSPLIT.NOTUSED;
                            break;
                    }

                    if (article.Attributes["TOPICS"].Value == "YES") { hasTopics = true; }
                    else { hasTopics = false; }

                    XmlNodeList articleData = article.ChildNodes;
                    foreach (XmlNode data in articleData)
                    {
                        switch (data.Name)
                        {
                            case "DATE":
                                date = data.InnerText;
                                break;
                            case "TOPICS":
                                if (data.HasChildNodes)
                                {
                                    topics = new List<string>();
                                    foreach (XmlNode topic in data.ChildNodes)
                                    {
                                        topics.Add(topic.InnerText);
                                    }
                                }
                                break;
                            case "PLACES":
                                if (data.HasChildNodes)
                                {
                                    places = new List<string>();
                                    foreach (XmlNode place in data.ChildNodes)
                                    {
                                        places.Add(place.InnerText);
                                    }
                                }
                                break;
                            case "TEXT":
                                if (data.HasChildNodes)
                                {
                                    foreach (XmlNode item in data.ChildNodes)
                                    {
                                        switch (item.Name)
                                        {
                                            case "AUTHOR":
                                                author = item.InnerText;
                                                break;
                                            case "BODY":
                                                body = item.InnerText.ToLower();
                                                if (removeStopWords)
                                                {
                                                    //remove unnecessary chars with space
                                                    char[] bodyCharArr = body.ToCharArray();
                                                    
                                                    for (int j = 0; j < bodyCharArr.Length; j++)
                                                    {
                                                        if (wrongChars.Contains(bodyCharArr[j]))
                                                        {
                                                            bodyCharArr[j] = ' ';
                                                        }
                                                    }
                                                    body = new string(bodyCharArr);

                                                    //remove whole words
                                                    if (stopWords == null)
                                                        continue;

                                                    string[] bodyArr = body.Split(' ');
                                                    body = string.Empty;
                                                    foreach (string word in bodyArr)
                                                    {
                                                        if (word == " " || word == "")
                                                            continue;
                                                        if (!stopWords.Contains(word))
                                                        {
                                                            body += word + ' ';
                                                        }
                                                        else
                                                        {
                                                        }
                                                    }
                                                }
                                                if (stem)
                                                {
                                                    Stemmer stemmer = new Stemmer();
                                                    string[] bodyArr = body.Split(' ');
                                                    body = string.Empty;
                                                    foreach (string word in bodyArr)
                                                    {
                                                        stemmer.add(word.ToCharArray(), word.Length);
                                                        stemmer.stem();
                                                        body += stemmer.ToString() + " ";
                                                    }
                                                }
                                                break;
                                            case "TITLE":
                                                title = item.InnerText;
                                                break;
                                        }
                                    }
                                }
                                break;
                        }
                    }
                    dataset.Add(new Article(id, date, author, body.Trim(), title, topics, places, fi.Name, hasTopics, lewissplit));
                }//for each article
            }//for each file
            return dataset;
        }

        private static string[] LoadStopWords()
        {
            StreamReader reader = new StreamReader(@"..\..\stopwords.txt");
            List<string> stopwords = new List<string>();
            while (!reader.EndOfStream)
            {
                stopwords.Add(reader.ReadLine());
            }
            reader.Close();
            return stopwords.ToArray();
        }

        /// <summary>
        /// This will return the articles that are in the wanted topics list AND have only one topic
        /// </summary>
        /// <param name="dataset"></param>
        /// <returns></returns>
        public static Dictionary<string, List<ICheckable>> GetWantedTopicsSingleItems(List<ICheckable> dataset)
        {
            Dictionary<string, List<ICheckable>> ans = new Dictionary<string, List<ICheckable>>();
            Article article = null;
            List<string> wantedTopics = new List<string>()
            {
                "gold","ship","sugar","crude","cocoa","copper","cotton","jobs","gnp","livestock"
            };

            //look for dicuments with one topic
            foreach (var item in dataset)
            {
                if (item.GetType() == typeof(Article))
                {
                    article = (Article)item;
                    if (article.Topics != null &&
                        article.Topics.Count == 1 &&
                        wantedTopics.Contains(article.Topics[0]))
                    {
                        if (ans.ContainsKey(article.Topics[0]))
                        {
                            ans[article.Topics[0]].Add(item);
                        }
                        else
                        {
                            ans.Add(article.Topics[0], new List<ICheckable>(){item});
                        }
                    }
                }
            }
            return ans;
        }

    }
}